import fs from "fs";
import path from "path";
import puppeteer from "puppeteer";

const __dirname = path.dirname(new URL(import.meta.url).pathname);

//打开浏览器，进入谷歌翻译网页
const browser = await puppeteer.launch({
  headless: false,
  // defaultViewport: {
  //   width: "100%",
  //   height: "100%",
  // },
  ignoreDefaultArgs: ["--enable-automation"],
  // devtools: true,
  slowMo: 300,
});
const pageFn = async (url) => {
  const page = await browser.newPage();
  await page.evaluateOnNewDocument(() => {
    Object.defineProperty(navigator, "webdriver", { get: () => false });
  });
  await page.goto(url, {
    waitUntil: "load",
    timeout: 600000,
  });
  return page;
};
const pcWeb = async () => {
  const page = await pageFn("https://www.ccdi.gov.cn/");
  const inputText = await page.$("#searchword");
  await inputText.type("案例剖析");
  const newPagePromise = new Promise((res) =>
    browser.once("targetcreated", (target) => res(target.page()))
  );
  await page.click(".close");
  await page.click(".but_search");
  let pageIndex = 1;
  let pageTotal = 0;
  const fn = async () => {
    const newPage = await newPagePromise;
    try {
      await newPage.waitForSelector(".page");
    } catch {
      await newPage.waitForSelector(".page");
    }
    const data = await newPage.evaluate(async () => {
      const info = { contentList: [], totalPage: 0 };
      const a = document.querySelectorAll(".s_0603_list li");
      for await (let item of a) {
        const time = item.querySelector(".time")?.textContent;
        if (
          new Date(time.match(/\d+-\d+-\d+/)?.[0]).getTime() >=
          new Date("2012-11-08").getTime()
        ) {
          //   const content = await pageFn(item.querySelector("a")?.href);
          //   await content.waitForSelector(".tit");
          //   const title = await content.$(".tit")?.textContent;
          info.contentList.push(item.querySelector("a")?.href);
          //   fs.writeFileSync(`./tools/pc/content/${title}`, title);
          //   const contentData = await newPage.evaluate(() => {})
        }
      }
      if (document.querySelector(".last-page")) {
        info.totalPage = Number(
          document.querySelector(".last-page")?.href?.match(/page=(\d+)&/)?.[1]
        );
      }

      return info;
    });

    if (data?.totalPage !== 0) {
      pageTotal = data.totalPage;
    }
    fs.appendFileSync("./tools/pc/url.txt", data.contentList.join("\n"));
    pageIndex++;
    console.log(`进度：${pageIndex - 1}/${pageTotal}完成！`);
    if (pageTotal !== 0 && pageIndex > pageTotal) {
      return;
    }
    await newPage.click(".next-page");
    await fn();
  };
  await fn();
  console.log("lfsz 链接爬取完成");
};
// pcWeb();
const getContent = async () => {
  const data = fs.readFileSync("./tools/pc/url.txt", "utf-8");
  const contentList = data.split("\n");
  for await (let x of contentList) {
    const page = await pageFn(x);
    await page.waitForSelector(".tit");
    const pageInfo = await page.evaluate(() => {
      const list = [];
      document
        .querySelectorAll(".content p")
        .forEach((item) => list.push(item.textContent));
      const info = {
        title: document
          .querySelector(".tit")
          ?.textContent?.trim()
          ?.replace("\n", ""),
        time: document
          .querySelector(".e2")
          ?.textContent.match(/\d+-\d+-\d+/)?.[0],
        content: list,
      };
      return info;
    });
    fs.writeFileSync(
      `./tools/pc/content/[${pageInfo.time}]${pageInfo.title}.txt`,
      pageInfo.content.join("\n")
    );
    console.log("lfsz", pageInfo);
  }
};

getContent();
